import json
import os
import random
import re
import time

from fake_useragent import UserAgent
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.options import Options
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from sortedcontainers import SortedSet

# 设置 Edge WebDriver 的路径
edge_driver_path = "C:/xiaoyh/edge_driver/msedgedriver.exe"

# 设置 Edge 浏览器的选项
options = Options()
options.add_argument('--headless')  # 无头模式
options.add_argument('--disable-gpu')  # 禁用GPU加速
options.add_argument('--no-sandbox')  # 解决权限问题
# 使用随机 User-Agent
ua = UserAgent()
options.add_argument(f"user-agent={ua.random}")

# 启动 Microsoft Edge 浏览器
driver = webdriver.Edge(service=Service(edge_driver_path), options=options)


# 定义动态等待方法
def wait_for_elements(context, by, value, timeout=10):
    """等待指定元素加载完成"""
    return WebDriverWait(context, timeout).until(
        EC.presence_of_all_elements_located((by, value))
    )


def wait_for_element(context, by, value, timeout=10):
    """等待指定元素加载完成"""
    return WebDriverWait(context, timeout).until(
        EC.presence_of_element_located((by, value))
    )


# 定义点击 '下一页' 按钮的方法
def get_next_page_link(driver):
    """获取 '查看更多' 按钮的链接"""
    try:
        next_page = driver.find_element(By.CLASS_NAME, "amore")
        next_page_link = next_page.get_attribute("href")
        print(f"下一页链接: {next_page_link}")
        return next_page_link
    except Exception as e:
        print("无法找到 '查看更多' 按钮:", e)
        return None

def handle_content(content):
    """处理内容"""
    # 去除数字、标点符号
    content = re.sub(r'[\d、]', '', content).strip()
    content = content.replace("(名言网)", "").replace("摘自名言网(www.mingyannet.com)", "")
    return content.strip()

def scrape_page(driver, data_list, title_list):
    """抓取当前页面数据并返回下一页链接"""
    title = wait_for_element(driver, By.TAG_NAME, "h1")
    current_title = title.text.strip()
    print(f"标题: {current_title}")

    if current_title in title_list:
        return None  # 如果标题重复，停止抓取
    title_list.append(current_title)

    # 获取内容
    txt_box = wait_for_element(driver, By.CLASS_NAME, "txt")
    content_elements = txt_box.find_elements(By.TAG_NAME, "p")

    # 删除不需要的内容
    remove_elements = txt_box.find_elements(By.TAG_NAME, "strong")
    content_elements = [
        el for el in content_elements if el not in remove_elements and el.text.strip() != current_title
    ]

    # 输出当前页面数据
    print(f"{current_title} 页面, 共 {len(content_elements)} 条数据")
    for content in content_elements:
        content_text = content.text.strip()
        if content_text:
            data_list.add(content_text)

    # 获取下一页链接
    try:
        page_box = txt_box.find_element(By.CLASS_NAME, "pager2")
        next_page_link = page_box.find_elements(By.TAG_NAME, "a")[-1].get_attribute("href")
        return next_page_link
    except Exception:
        print("未找到下一页链接")
        return None

def save_data(data_list, output_dir, module_name):
    """保存抓取的数据为 JSON 文件"""
    os.makedirs(output_dir, exist_ok=True)
    filename = os.path.join(output_dir, f"{module_name}.json")
    with open(filename, "w", encoding="utf-8") as file:
        json.dump(
            [{"index": idx + 1, "content": handle_content(content)} for idx, content in enumerate(data_list)],
            file,
            ensure_ascii=False,
            indent=4,
        )
    print(f"模块 {module_name} 抓取成功，共 {len(data_list)} 条数据")

def main():
    url = "https://www.mingyannet.com/mingyan/254115913"
    module_name = "名言"
    output_dir = "output/selenium/mingyan"

    data_list = set()
    title_list = []

    try:
        driver.get(url)
        while True:
            next_page_link = scrape_page(driver, data_list, title_list)
            if not next_page_link or len(data_list) >= 10000:
                break
            driver.get(next_page_link)
            time.sleep(random.randint(1, 3))  # 限制请求速率

        if data_list:
            save_data(data_list, output_dir, module_name)
        else:
            print(f"模块 {module_name} 抓取失败，没有数据")
    except Exception as e:
        print("发生错误:", e)
    finally:
        driver.quit()

if __name__ == "__main__":
    main()